/* The ns package provides the net-score.org functionality. */
package ns

import (
	"appengine"
	"appengine/datastore"
	"appengine/memcache"
	"appengine/user"
	"bytes"
	"encoding/gob"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"strconv"
	"strings"
	"time"
)

var (
	ErrTestRecordMissing = errors.New("test record missing")
	ErrTestDataMissing   = errors.New("test data missing")
	ErrHttpHeaderMissing = errors.New("http header missing")
	ErrUrlParamMissing   = errors.New("URL parameter missing")

	//Used to tell the client not to look for data.
	InvalidData = []byte("{\"netscoreError\": [\"IvalidData\"]}")
)

func init() {
	//NOTE: depreciated 
	http.HandleFunc("/upload/", upload)
	http.HandleFunc("/upload", upload)
	//NOTE: depreciated 
	http.HandleFunc("/upload/JsError/", uploadJsError)
	http.HandleFunc("/upload/JsError", uploadJsError)
}

//NOTE: depreciated 
type UserListData struct {
	Key        string
	Date       int64
	RemoteAddr string
	TestForm   string
}

//NOTE: depreciated 
type ListData struct {
	Key            string
	Date           int64
	UserId         string
	DevKey         string
	VersionID      string
	RemoteAddr     string
	IndexVersionID string
	TestForm       string
}

//NOTE: depreciated 
type ServerData struct {
	Date           int64
	UserId         string
	DevKey         string
	VersionID      string
	RemoteAddr     string
	Header         map[string]string
	IndexVersionID string
	TestForm       string
}

//NOTE: depreciated 
type JsonReadyData struct {
	Server ServerData
	Client map[string]interface{}
}

/*Struct for holding test meta data.

One of these structs will be created for every test request.  That is a HTTP
GET request that should result in the client running code.  In the case where a
test fails, this struct will be incomplete and hopefully there will be something
in the TestError field.  When searching for data by anything other than GetTime
or RemoteAddr you should search on this and then use the TestDataKey to get to
the data.
*/
type TestRecord struct {
	RemoteAddr     string
	GetTime        int64
	PostTime       int64
	UserID         string
	GetHeader      []byte `datastore:",noindex"`
	PostHeader     []byte `datastore:",noindex"`
	Country        string
	Region         string
	City           string
	Latitude       float64
	Longitude      float64
	GetRecordKey   *datastore.Key `json:"-"`
	PostRecordKey  *datastore.Key `json:"-"`
	DevKey         string
	TestForm       TestFormat
	TestDataKey    *datastore.Key `json:"-"`
	AppVersionID   string
	IndexVersionID string
	TestError      string
	//ASN		string
	//ISP		string
	//ISPdomain	string
}

/*Returns a string suitable for use as a TestRecord key.

This is globally unique for all the TestRecord structs since it is unlikely that
two requests are handled from the same IP address at the same time.
*/
func (tr *TestRecord) String() string {
	return fmt.Sprintf("%d_%s", tr.GetTime, tr.RemoteAddr)
}

/*Creates a new TestRecord struct and fills in basic info.

Since the only time that a new TestRecord should be created is when we have a
new request then we can accept the context of the request and use that
information to pre-fill in as much as we can.  What is not completed is
information that will become known when the client posts the results and when
indexing server logs at a later time.

NOTE: look at the source to see what has been pre-filled.
		//GetRecordKey	*datastore.Key
		//PostRecordKey	*datastore.Key
*/
func NewTestRecord(c appengine.Context, r *http.Request, gt int64) *TestRecord {
	tr := &TestRecord{
		RemoteAddr:     r.RemoteAddr,
		GetTime:        gt,
		AppVersionID:   appengine.VersionID(c),
		IndexVersionID: "",
		TestError:      "",
	}

	//Set the location from GAE headers
	tr.Country = r.Header.Get("X-AppEngine-Country")
	tr.Region = r.Header.Get("X-AppEngine-Region")
	tr.City = r.Header.Get("X-AppEngine-City")
	latLong := strings.Split(r.Header.Get("X-AppEngine-CityLatLong"), ",")
	if len(latLong) == 2 {
		var err error
		tr.Latitude, err = strconv.ParseFloat(latLong[0], 64)
		if err != nil {
			//tr.Latitude is 0
			c.Errorf("NewTestRecord:strconv.ParseFloat err =  %v",
				err)
		}
		tr.Longitude, err = strconv.ParseFloat(latLong[1], 64)
		if err != nil {
			//tr.Longitude is 0
			c.Errorf("NewTestRecord:strconv.ParseFloat err =  %v",
				err)
		}
	}

	//Write the user if we have one.
	u := user.Current(c)
	if u != nil {
		tr.UserID = u.ID
		//TODO(gavaletz) put this in a taskqueue
		StoreUser(c, u)
	}

	//Flatten the header to store as a blob
	header := make([]string, len(r.Header))
	i := 0
	for k, v := range r.Header {
		header[i] = fmt.Sprintf("%s:%s", k, v[0])
		i += 1
	}
	tr.GetHeader = []byte(strings.Join(header, "\n"))

	return tr
}

/*Looks for and returns an existing TestRecord.

This is most commonly called when a client is ready to post data to the server.
It first checks memcache since it is likely that the item was just created when
the test code was requested.  In the event that it got bumped from the cache it
will check the datastore.  In the event that it is not found in the datastore it
will return an Error along with a BLANK TestRecord entity.  Since not having a
record of when the test was requested is in indicator of a problem we may want
to at least flag this data, but it is up to the called to proceed or not.
*/
func GetTestRecord(c appengine.Context, getTime int64, ip string) (*TestRecord,
	error) {
	tr := new(TestRecord)
	keyString := fmt.Sprintf("%d_%s", getTime, ip)

	//Check memcache
	item, err := memcache.Get(c, keyString)
	if err == nil {
		//create item
		b := bytes.NewBuffer(item.Value)
		d := gob.NewDecoder(b)
		err = d.Decode(tr)
		if err != nil {
			c.Errorf("GetTestRecord:gob.Decode for %s err =  %v",
				keyString, err)
		} else {
			return tr, nil
		}
	} else if err != memcache.ErrCacheMiss {
		c.Errorf("GetTestRecord:memcache.Get for %s err =  %v",
			keyString, err)
	}

	//create key
	key := datastore.NewKey(c, "TestRecord", keyString, 0, nil)

	//Check datastore 
	err = datastore.Get(c, key, tr)
	if err == datastore.ErrNoSuchEntity {
		return tr, ErrTestRecordMissing
	} else if err != nil {
		return tr, err
	}
	return tr, nil
}

/*Saves the TestRecord struct to memcache and backs it up with datastore.

Because a TestRecord is created when the test code is requested and then
updated when the test completes we can save the read cost of the datastore
entity and speed things up by using memecache.  Since an item can be kicked out
of memecache at any time it is important that we back it up by writing to the
datastore.
*/
func (tr *TestRecord) Cache(c appengine.Context) (*datastore.Key, error) {
	key := datastore.NewKey(c, "TestRecord", tr.String(), 0, nil)

	//save to memcache
	b := new(bytes.Buffer)
	e := gob.NewEncoder(b)
	err := e.Encode(tr)
	if err != nil {
		c.Errorf("Cache:gob.Encode for %s err =  %v", tr.String(), err)
	} else {
		item := &memcache.Item{
			Key:   tr.String(),
			Value: b.Bytes(),
		}
		err = memcache.Set(c, item)
		if err != nil {
			c.Errorf("Cache:memcache.Set for %s err =  %v",
				tr.String(), err)
		}
	}

	//save to datastore
	_, err = datastore.Put(c, key, tr)
	if err != nil {
		return key, err
	}
	return key, nil
}

/*Saves the TestRecord struct to the datastore.

This method exists because if it is unlikely that the entity will be read again
soon after the test completes.  We might evict an item from memcache that will
be needed soon if we write something we don't expect to need.  Since this is the
case we explicitly delete the item from the cache.
*/
func (tr *TestRecord) Save(c appengine.Context) (*datastore.Key, error) {
	key := datastore.NewKey(c, "TestRecord", tr.String(), 0, nil)

	//Make sure there is not an old value in the cache.
	memcache.Delete(c, tr.String())

	//save to datastore
	_, err := datastore.Put(c, key, tr)
	if err != nil {
		return key, err
	}
	return key, nil
}

//NOTE: depreciated 
/*Serializes a TestRecord object and its dependencies.
 */
func (tr *TestRecord) MarshalJSON(c appengine.Context) ([]byte, error) {
	var unmarshaledData interface{}
	var err error

	//Get the TestData
	td := new(TestData)
	//if tr.Valid {
	if tr.TestDataKey != nil {
		err = datastore.Get(c, tr.TestDataKey, td)
		_, ok := err.(*datastore.ErrFieldMismatch)
		if ok {
			//We are ignoring the datastore.ErrFieldMismatch error
			c.Errorf("MarshalJSON:datastore.Get for %s err =  %v",
				tr.String(), err)
			err = nil
		}
		if err != nil {
			return nil, err
		}

		err = json.Unmarshal(td.RawData, &unmarshaledData)
		if err != nil {
			return nil, err
		}
	} else {
		err = json.Unmarshal(InvalidData, &unmarshaledData)
		if err != nil {
			return nil, err
		}
	}

	sd := ServerData{
		Date:           tr.GetTime,
		UserId:         tr.UserID,
		DevKey:         tr.DevKey,
		VersionID:      tr.AppVersionID,
		RemoteAddr:     tr.RemoteAddr,
		IndexVersionID: tr.IndexVersionID,
		TestForm:       tr.TestForm.String(),
	}

	//Fill in the header.  This takes care of a problem that occurs when you
	// have an incomplete test that lacks a PUT header.
	var lines []string
	if tr.TestDataKey != nil {
		sd.Header = make(map[string]string,
			strings.Count(string(tr.PostHeader), ":"))
		lines = strings.Split(string(tr.PostHeader), "\n")
	} else {
		sd.Header = make(map[string]string,
			strings.Count(string(tr.GetHeader), ":"))
		lines = strings.Split(string(tr.GetHeader), "\n")
	}
	for s := range lines {
		rec := strings.Split(lines[s], ":")
		sd.Header[strings.Trim(rec[0], " \n\r\t")] = strings.Trim(
			strings.Join(rec[1:], ":"), " \n\r\t")
	}

	jd := JsonReadyData{
		Server: sd,
		Client: unmarshaledData.(map[string]interface{}),
	}

	var JSONdata []byte
	JSONdata, err = json.Marshal(jd)
	if err != nil {
		return JSONdata, err
	}
	return JSONdata, nil
}

//NOTE: depreciated 
/*Serializes a TestRecord object and its dependencies.
 */
func (tr *TestRecord) MarshalListUserJSON() ([]byte, error) {
	ld := UserListData{
		Key:        tr.String(),
		Date:       tr.GetTime,
		RemoteAddr: tr.RemoteAddr,
		TestForm:   tr.TestForm.String(),
	}

	var JSONdata []byte
	var err error
	if JSONdata, err = json.Marshal(ld); err != nil {
		return JSONdata, err
	}
	return JSONdata, nil
}

//NOTE: depreciated 
/*Serializes a TestRecord object and its dependencies.
 */
func (tr *TestRecord) MarshalListJSON() ([]byte, error) {
	ld := ListData{
		Key:            tr.String(),
		Date:           tr.GetTime,
		UserId:         tr.UserID,
		DevKey:         tr.DevKey,
		VersionID:      tr.AppVersionID,
		RemoteAddr:     tr.RemoteAddr,
		IndexVersionID: tr.IndexVersionID,
		TestForm:       tr.TestForm.String(),
	}

	var JSONdata []byte
	var err error
	if JSONdata, err = json.Marshal(ld); err != nil {
		return JSONdata, err
	}
	return JSONdata, nil
}

/*Struct for holding the test results.

One of these should be created for every post.  When data is first posted, it
is important that at least the RawData is saved.  It is possible to fill in
other values during the indexing process.

NOTE: If you add new fields here, be sure to update the indexing functions so
that they can be filled in as needed.  It would also make sense to have a FixIt
function to fill the values in for entities with older IndexVersionIDs.
*/
type TestData struct {
	GetTime        int64
	RemoteAddr     string
	RawData        []byte `datastore:",noindex"`
	IndexVersionID string
	TestForm       TestFormat
	DataError      string
	//MedianLatency		float64
	//MedianThroughput	float64
	//ConnType		ConnectionType
	//LocType		LocationType
}

/*Returns a string suitable for use as a TestData key.

This is globally unique for all the TestData structs since it is unlikely that
two requests are handled from the same IP address at the same time.
*/
func (td *TestData) String() string {
	return fmt.Sprintf("%d_%s", td.GetTime, td.RemoteAddr)
}

/*Creates a new TestData struct and fills it in.

Since the only time that a new TestData struct should be created is when a
client has posted data.  With the request then we can accept the request and use
that information to pre-fill the struct.  What is not completed should be filled
in during the indexing process.

NOTE: we are not filling in the TestForm since that info should be in the
	TestRecord.
*/
func NewTestData(c appengine.Context, r *http.Request, getTime int64) (
	*TestData, error) {
	td := &TestData{
		GetTime:        getTime,
		RemoteAddr:     r.RemoteAddr,
		RawData:        make([]byte, r.ContentLength),
		IndexVersionID: "",
		DataError:      "",
	}

	//Read the body if we have one
	if r.ContentLength > 0 {
		//If you don't use ReadFull you only get 3316 Bytes
		bytesRead, err := io.ReadFull(r.Body, td.RawData)
		if err != nil {
			c.Errorf("NewTestData:io.ReadFull err = %v", err)
			return td, err
		}
		if int64(bytesRead) != r.ContentLength {
			c.Warningf("NewTestData:io.ReadFull only read %d of %d bytes",
				bytesRead, r.ContentLength)
		}
	}
	return td, nil
}

/*Looks for and returns an existing TestData struct.

This is most commonly called when a client is ready to post data to the server.
In the event that it is not found in the datastore it will return an Error along
with a BLANK TestData struct.
*/
func GetTestData(c appengine.Context, getTime int64, ip string) (*TestData, error) {
	td := new(TestData)
	keyString := fmt.Sprintf("%d_%s", getTime, ip)

	//create key
	key := datastore.NewKey(c, "TestData", keyString, 0, nil)

	//Check datastore 
	err := datastore.Get(c, key, td)
	if err == datastore.ErrNoSuchEntity {
		return td, ErrTestDataMissing
	} else if err != nil {
		c.Errorf("GetTestData:datastore.Get for %s err =  %v",
			keyString, err)
		return td, err
	}
	return td, nil
}

/*Saves the TestData to the datastore.

Since this is generally an infrequently used item we don't think that it is
worth possibly kicking other items out of the datastore to accommodate this.
*/
func (td *TestData) Save(c appengine.Context) (*datastore.Key, error) {
	key := datastore.NewKey(c, "TestData", td.String(), 0, nil)
	if _, err := datastore.Put(c, key, td); err != nil {
		c.Errorf("Save:datastore.Put err = %v", err)
		return key, err
	}
	return key, nil
}

/* The RawData struct holds all of the data pre-processing.

NOTE: This is deprecated.
*/
type RawData struct {
	Date           int64
	UserId         string
	RawURL         string
	DevKey         string
	Host           string
	VersionID      string
	RemoteAddr     string
	Body           []byte `datastore:",noindex"`
	Header         []byte `datastore:",noindex"`
	Indexed        bool
	IndexVersionID string
	TestForm       TestFormat
}

//NOTE: This is deprecated.
func (rd *RawData) String() string {
	return fmt.Sprintf("%d_%s", rd.Date, rd.RemoteAddr)
}

//NOTE: This is deprecated.
func (rd *RawData) Save(c appengine.Context) {
	key := datastore.NewKey(c, "RawData", rd.String(), 0, nil)
	if _, err := datastore.Put(c, key, rd); err != nil {
		c.Errorf("Save:datastore.Put err = %v", err)
	}
}

/*Writes the User entity to our datastore.

This never needs to be put in memcache because we never need to read it.
There may be a small gain to be made by putting the key in memcache, but that
only saves 1 Small Read on a hit.
*/
func StoreUser(c appengine.Context, u *user.User) {
	key := datastore.NewKey(c, "User", u.ID, 0, nil)
	eu := new(user.User)
	if err := datastore.Get(c, key, eu); err == datastore.ErrNoSuchEntity {
		if _, err := datastore.Put(c, key, u); err != nil {
			c.Errorf("StoreUser:datastore.Put err = %v", u, err)
		}
		c.Infof("WELCOME NEW USER = %s", u)
	}
}

/*HTTP handler for GET requests with JavaScript errors.

This method should translate errors in the JavaScript clients into errors in the
GAE server logs.
*/
func uploadJsError(w http.ResponseWriter, r *http.Request) {
	jsErrorKey := r.FormValue("k")
	jsErrorValue := r.FormValue("v")
	c := appengine.NewContext(r)
	c.Errorf("JsError: %v = %v", jsErrorKey, jsErrorValue)
	fmt.Fprint(w, "OK")
}

/*HTTP handler for POST requests with data.

This method should accept data that is being POSTed as a response to a test
request.  It is important that the client made a request for the test before
trying to upload data.  If this is not the case then we will reject the data as
a potential source of spam.  The method will update an existing TestRecord
struct and create a new TestData struct.  The response to the client should be
the key for the test record in a string format.
*/
func upload(w http.ResponseWriter, r *http.Request) {
	pt := time.Now().Unix()
	c := appengine.NewContext(r)

	//Get the request time from the HTTP header.
	var gt int64
	var err error
	requestTime := r.FormValue("gt")
	if requestTime != "" {
		gt, err = strconv.ParseInt(requestTime, 10, 64)
		if err != nil {
			c.Errorf("upload:strconv.ParseInt err = %v", err)
			http.Error(w, err.Error(),
				http.StatusInternalServerError)
			return
		}
	} else {
		c.Errorf("upload:r.FormValue err = %v", ErrUrlParamMissing)
		http.Error(w, ErrUrlParamMissing.Error(),
			http.StatusInternalServerError)
		return
	}

	//Get the existing TestRecord
	tr, err := GetTestRecord(c, gt, r.RemoteAddr)
	if err != nil {
		c.Errorf("upload:GetTestRecord err = %v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

	tr.PostTime = pt

	//Flatten the header to store as a blob.
	header := make([]string, len(r.Header))
	i := 0
	for k, v := range r.Header {
		header[i] = fmt.Sprintf("%s:%s", k, v[0])
		i += 1
	}
	tr.PostHeader = []byte(strings.Join(header, "\n"))

	//Create the testData and save it and the key in the TestRecord.
	td, err := NewTestData(c, r, gt)
	if err != nil {
		c.Errorf("upload:NewTestData err = %v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	td.TestForm = tr.TestForm

	tr.TestDataKey, err = td.Save(c)
	if err != nil {
		c.Errorf("upload:td.Save err = %v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	_, err = tr.Save(c)
	if err != nil {
		c.Errorf("upload:tr.Save err = %v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	fmt.Fprint(w, tr.String())
}
